package per.hnvcam.searchext.text.token;

/**
 * Created by IntelliJ IDEA.
 * User: violon
 * Date: 4/24/11
 * Time: 6:00 PM
 * To change this template use File | Settings | File Templates.
 */
public interface Tokenizer {
    void setWordDelimeter(char wordDelimeter);
    void setPhraseDelimeters(String phraseDelimeters);
    void lex(String input);
    void setHandler(TokenHandler handler);
}
